import requests
import pymongo
import time
import log_helper
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

myclient = pymongo.MongoClient("mongodb://82.157.127.245:27999/")
mydb = myclient["weibo"]
mycollection = mydb["hotsearch"]

request_headers = {
    "accept": "application/json, text/plain, */*",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
    "referer": "https://weibo.com/u/7203149652/home?wvr=5",
    "connection": "close",
    "cookie": "SINAGLOBAL=5188697282285.41.1591635317397; UOR=,,login.sina.com.cn; ULV=1636468001389:24:8:5:1590000126304.5515.1636468001346:1636459251317; SCF=Ai2Z0Av6-LNog9gdRY2nsteOJC6e2CE2SpTKVU8Z4qEGHXN0NXAB79vM_Jy6DDyizTCTsqVUjcNBlV7ygGvSuMs.; XSRF-TOKEN=IE6RL7a1qqQVFrs90nowUSWC; SUB=_2A25MivpkDeRhGeNG71QU9S_NyTuIHXVv_mysrDV8PUNbmtANLRfFkW9NS0WHTG4z5fv8Metu5YgAGLznMQhPYzPL; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWMpsjhFWGvDNhUl8gSQh155JpX5KzhUgL.Fo-RShqfSK2peoM2dJLoIpeLxKML1--LB-80IsHV9c8790e0; ALF=1668267444; SSOLoginState=1636731444; WBPSESS=Dt2hbAUaXfkVprjyrAZT_O7srNv1hiaxzlKI2VfifSBrOS_Y6N9Waix4b-nj-gG_A8aSCX62-q73R6VDEUSj_GzGalsItrGOKHROTxKudWDJHEXkVQt1Vr32p9u16Pi_XZq_CvXeLnoOdSniiZb5o8fco2M2APxGsZk4YdgXEaQi9r04P4Ij76osmnaUPeVJeLYqawgsiT3N8Vx9_IfI0w=="
}
#hotSearch:get

hot_search_url = "https://weibo.com/ajax/side/hotSearch"

hot_search_resp = requests.session().get(hot_search_url, headers=request_headers, verify=False)
realtime_list = hot_search_resp.json().get('data').get('realtime')

search_url = "https://weibo.com/ajax/search/all"

time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

for realtime in realtime_list:
    #try:
    event = {
        "word": realtime.get("word"),
        "category": realtime.get("category"),
        "rank": realtime.get("rank") + 1,
        "time": time_str,
        "num": realtime.get("num"),
        "emoticon": realtime.get("emoticon"),
    }
    page = 1
    blog_cards = []
    print("正在获取话题#{}#的相关微博".format(realtime.get("word")))
    while True:

        params = {
            "containerid": "100103type=1&q=#" + realtime.get("word") + "#&t=3",
            "page": page,
            "count": 20
        }
        status = -1

        try:
            search_resp = requests.session().get(search_url, headers=request_headers, params=params, verify=False)
            status = search_resp.json().get("ok")
        except Exception as e:
            log_helper.error("话题#{}#的第{}页获取失败".format(realtime.get("word"), page))
            continue


        if status < 0:
            log_helper.error("cookies失效")
            break;
        if search_resp.json().get("data") is None:
            continue

        cards = search_resp.json().get("data").get("cards")

        if len(cards) == 0:
            log_helper.error("话题#{}#评论获取完成".format(realtime.get("word")))
            break;
        for card in cards:
            blog_card = {}
            if card["card_type"] == 7:
                #blog["is_hotweibo"] = card.get("is_hotweibo",0)
                event["desc"] = card.get("desc")
            pic_list = []
            if card["card_type"] == 9:
                blog_card["text_raw"] = card.get("mblog").get("text_raw")
                blog_card["is_hotweibo"] = card.get("is_hotweibo", 0)
                if card.get("mblog").get("source") == "微博视频号":
                    blog_card["is_video"] = 1
                    if card.get("mblog").get("url_struct") is None:
                        blog_card["video_url"] = "获取视频链接失败"
                    else:
                        blog_card["video_url"] = card.get("mblog").get("url_struct")[0].get("short_url")
                    if card.get("mblog").get("page_info") is None:
                        blog_card["video_cover"] = "获取图片信息失败"
                    else:
                        blog_card["video_cover"] = card.get("mblog").get("page_info").get("page_pic")
                else:
                    if card.get("mblog").get("url_struct") is not None:
                        long_url = card.get("mblog").get("url_struct")[0].get("long_url")
                        if long_url.find("video.weibo.com") != -1 and card.get("mblog").get("page_info") is not None:
                            blog_card["is_video"] = 1
                            blog_card["video_url"] = long_url
                            blog_card["video_cover"] = card.get("mblog").get("page_info").get("page_pic")
                        else:
                            blog_card["is_video"] = 0
                            blog_card["video_url"] = ""
                            blog_card["video_cover"] = ""
                    else:
                        blog_card["is_video"] = 0
                        blog_card["video_url"] = ""
                        blog_card["video_cover"] = ""

                pic_infos = card.get("mblog").get("pic_infos", {})
                pic_ids = card.get("mblog").get("pic_ids", [])

                for pic_id in pic_ids:
                    if pic_infos.get(pic_id) is None:
                        continue
                    if pic_infos.get(pic_id).get("original") is None:
                        print("original is None", realtime.get("word"))

                    pic_list.append(pic_infos.get(pic_id).get("original").get("url"))

                blog_card["user"] = card.get("mblog").get("user")
                blog_card["pic_list"] = pic_list
                blog_card["create_time"] = card.get("mblog").get("created_at")
            if card["card_type"] == 11:
                if card.get("card_group")[0].get("card_type") == 10:
                    continue
                if card.get("card_group")[0].get("card_type") == 9:
                    blog_card["is_hotweibo"] = card.get("card_group")[0].get("is_hotweibo", 0)
                    blog_card["text_raw"] = card.get("card_group")[0].get("mblog").get("text_raw")

                    pic_infos = card.get("card_group")[0].get("mblog").get("pic_infos",{})
                    pic_ids = card.get("card_group")[0].get("mblog").get("pic_ids",[])
                    for pic_id in pic_ids:
                        if pic_infos.get(pic_id) is None:
                            continue
                        pic_list.append(pic_infos.get(pic_id).get("original").get("url"))

                    if card.get("card_group")[0].get("mblog").get("source") == "微博视频号":
                        blog_card["is_video"] = 1

                        if card.get("card_group")[0].get("mblog").get("url_struct") is None:
                            blog_card["video_url"] = "获取视频链接失败"
                        else:
                            blog_card["video_url"] = card.get("card_group")[0].get("mblog").get("url_struct")[0].get("short_url")

                        if card.get("card_group")[0].get("mblog").get("page_info") is None:
                            blog_card["video_cover"] = "获取图片信息失败"
                        else:
                            blog_card["video_cover"] = card.get("card_group")[0].get("mblog").get("page_info").get("page_pic")
                    else:
                        if card.get("card_group")[0].get("mblog").get("url_struct") is not None:
                            long_url = card.get("card_group")[0].get("mblog").get("url_struct")[0].get("long_url")
                            if long_url.find("video.weibo.com") != -1 and card.get("card_group")[0].get("mblog").get("page_info") is not None:
                                blog_card["is_video"] = 1
                                blog_card["video_url"] = long_url
                                blog_card["video_cover"] = card.get("card_group")[0].get("mblog").get("page_info").get("page_pic")
                            else:
                                blog_card["is_video"] = 0
                                blog_card["video_url"] = ""
                                blog_card["video_cover"] = ""
                        else:
                            blog_card["is_video"] = 0
                            blog_card["video_url"] = ""
                            blog_card["video_cover"] = ""


                    blog_card["user"] = card.get("card_group")[0].get("mblog").get("user")
                    blog_card["create_time"] = card.get("card_group")[0].get("mblog").get("created_at")
                    blog_card["pic_list"] = pic_list
            if card["card_type"] != 7:
                blog_cards.append(blog_card)
        page += 1
    if len(blog_cards) != 0 :
        event["blog_cards"] = blog_cards
        if event.get("desc") is None:
            event["desc"] = realtime.get("word")
        mycollection.insert_one(event)

    # except Exception as e:
    #     log_helper.error(e)
    #     log_helper.error("话题#{}#评论获取失败".format(realtime.get("word")))

log_helper.error("{} 完成一次爬虫".format(time_str))
